package com.spider.ask120.magic.WebMagic;

import java.util.HashMap;
import java.util.List;
import java.util.Map;


import com.wilddog.client.Wilddog;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康经验 频道
 * @author Jxw
 *
 */
public class Jingyan  implements PageProcessor{
	static String jkjyUrl = "http://m.120ask.com/jingyan/";
	static String listUrl = "http://m.120ask.com/jingyan/list/\\w+"; 
	
	Wilddog dog = new Wilddog("https://testask.wilddogio.com");
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
 
	public void process(Page page) { 
		//进入频道页,获取各病种列表页URL
		if(page.getUrl().regex(jkjyUrl+"$").match()) {
			//保存经验频道
			dog.setValue("jingyan");
			dog.child("jingyan/value").setValue("健康经验");
			//获取到话题页列表URL 
			List<String> huati = page.getHtml().links().regex(listUrl).all(); 
			page.addTargetRequests(huati);
		 
		}else if(page.getUrl().regex(listUrl).match()) {//不包括.html后缀  //+"$" 
			String catalog = page.getRequest().getExtra("cat") == null ? "" : page.getRequest().getExtra("cat").toString();

			String cataName = page.getHtml().$("#classname","text").get();
			//从URL中获取当前目录名称
			String path = page.getUrl().toString();
			String key = "";
			if (path.indexOf(".html") > 0) {
				key = path.substring(path.indexOf("list/")+5, path.indexOf(".html"));
			} else {
				key = path.substring(path.indexOf("list/")+5);
			} 

			//保存目录数据结构到野狗
			if(catalog.isEmpty()){ 
				
				Wilddog jingyan = dog.child("jingyan/child/");
				jingyan.child(key+"/value").setValue(cataName);
			}else{
				String[] cats = catalog.split(",");
				String dog_path = "jingyan/child/";
				for(int i = 0; i < cats.length; i++){ 
					String cat = cats[i];
					
					if(i < cats.length){
						dog_path += cat+"/child/";
						
					}else{
						dog_path += cat+"/value/";
					}
					Wilddog jingyan = dog.child(dog_path);
					jingyan.setValue(cataName);
				}
			}
			
			//判断该页面中如果还有 下一级 栏目， 则继续获取连接
			if(page.getHtml().$(".s_content").match()){
				List<String> huati = page.getHtml().$(".s_content").links().regex(listUrl).all(); 
				for(String url : huati){ 
					Map<String, Object> extras = new HashMap<String, Object>();
					extras.put("cat", catalog.isEmpty() ? (""+key) : (","+key));
					
					Request request = new Request(url);
					request.setExtras(extras);
					page.addTargetRequest(request);
				}
			}
			//保存
//			else{
//				//无下级栏目，开始加载分页数据列表，获取正文URL
//				Map<String, String> params = new HashMap<String, String>();
//				params.put("ajax", "1");
//				int pageIndex = 1;	
//				while(true){ 
//					//从第一页开始取， 只能拿到HTML页面内容，取不到json格式的数据，所以跳过第一页；
//					pageIndex ++;
//					params.put("page", pageIndex+"");
//					String listJson = HttpClientUtil.getInstance().sendHttpPost(page.getUrl().toString(), params);
//					if(listJson != null && !listJson.isEmpty()){  
//						try {
//							JsonObject rstObj = (JsonObject) JsonUtil.parse(listJson); 
//							//{"code":200,"list":[]}
//							if(rstObj != null && rstObj.get("code") != null && "200".equals(rstObj.get("code").toString()) 
//											&& rstObj.get("list") != null && !"[]".equals(rstObj.get("list").toString())){
//								
//								JsonArray objAry = (JsonArray) JsonUtil.parse(rstObj.get("list").toString());
//								if(objAry.size() > 0){ 
//									for(int i = 0; i < objAry.size(); i++){
//										JsonObject obj = (JsonObject) objAry.get(i);
//										String id = obj.get("id").toString();
//										String id_str = obj.get("id_str").toString();
//										String title = obj.get("title").toString();
//										String content = obj.get("content").toString();
//										String pic = obj.get("pic").toString();
//									}
//								}else{ 
//									break;
//								}
//							}else{ 
//								break;
//							}
//						} catch (IOException e) { 
//							e.printStackTrace();
//						} catch (ParseException e) { 
//							e.printStackTrace();
//						}
//					}
//
//				}
//			}
			
			 
		}else if(page.getUrl().regex(jkjyUrl+"\\w+.html").match()) {
			page.putField("title", page.getHtml().xpath("//div[@class='g-jinyan-topic']/h3/text()")); 
			page.putField("create_time", page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").regex("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}"));
			String share = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").toString();
			page.putField("share", share.substring(share.indexOf("由")+1, share.indexOf("分享")).trim());
			page.putField("keyword", page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[2]/a/text()").all());
			//介绍
			page.putField("abstract", page.getHtml().xpath("//div[@class='g-jinyan-cont']/p/text()"));
			//步骤方法
			page.putField("pageContent", page.getHtml().xpath("//div[@class='d1 img']/dl/dd/html()").all());
			//注意事项
			page.putField("remind", page.getHtml().xpath("//div[@class='g-jinyan-conf']/p[2]/html()"));
			
		}
		
		
		

	}
  
	public static void main(String[] args) {
		Spider.create(new Jingyan()).addUrl("http://m.120ask.com/jingyan/list/1kwvmhsksk.html").thread(1).run(); 
		
    }
 
}
